
import glob
import re
import sys
import string
import pepe.p_txt.bigtxt as bt

space_fill = lambda x, y: x + ((y - len(x)) * ' ')

def main(input_file_mask):
    """ """
    for filename in glob.glob(input_file_mask):

        o = {}
    
        fname = string.split(filename, '.')[0]
        print fname
    
        for line in bt.lines(filename):
            line = re.sub('\x0D\x0A', '', line)
            line = re.sub('\x0D', '', line)
            line = re.sub('\x0A', '', line)

            # inspect full row
            mask = line[:]

            if 0:
                mask = re.sub('[A-Z]', 'S', mask)
                mask = re.sub('[a-z]', 's', mask)
                mask = re.sub('\d', 'N', mask)

            if 1:
                mask = re.sub('[A-Z]', 'X', mask)
                mask = re.sub('[a-z]', 'X', mask)
                mask = re.sub('\d', 'X', mask)
                mask = re.sub('[-/&(),$|]', 'X', mask)

            if o.has_key(mask):
                o[mask][0] += 1
                o[mask][1] = line[:]
            else:
                o[mask] = [1, '']
                o[mask][1] = line[:]

        # 2010-10-22_2257 vp
        ## using underscore instead dot because .# utilities 
        ## cut out everything after dot and then is not clear 
        ## whether utility was run on original file or on output
        ## of other .# utility that processed given file
        of = open(fname + '_rowsmask.#','wb')
        of.write('type\tobservations\t_\tmask_or_example\t_\t_\t\x0D\x0A'.upper())

        for k in sorted(o):
            #print space_fill(str(res[k]), 20), "\t:\t", k
            of.write(space_fill('line_mask', 20) + '\t' + space_fill(str(o[k][0]), 20) + "\t:\t" + k + '\x0D\x0A')
            of.write(space_fill('line_example', 20) + '\t' + space_fill(str(o[k][0]), 20) + "\t:\t" + o[k][1] + '\x0D\x0A')
    
        of.close()

if __name__ == '__main__':
    """ """
    if len(sys.argv) == 2:
        main(sys.argv[1])
    else:
        
        print """

        rowsmask <input-file-mask>

        example : rowsmask *.dat

        input-file-mask (mandatory)
            input data file(s) to be processed

        """